package com.dexcoder.job.baidu;

import com.alibaba.fastjson.JSON;
import com.dexcoder.demo.utils.JdbcUtils;
import com.dexcoder.demo.utils.MailUtils;
import com.dexcoder.demo.vo.CategoryVo;
import com.dexcoder.demo.vo.KeywordVo;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.stereotype.Component;


import javax.mail.*;
import javax.mail.internet.InternetAddress;
import javax.mail.internet.MimeMessage;
import java.io.IOException;
import java.sql.SQLException;
import java.util.*;

/**
 * Created by minniezhou on 2017/5/21.
 */
@Component
public class SogouSpider {

    public static void getSogouKeyWord(){
        List<KeywordVo> keywords = getKeyword();
        JdbcUtils jdbcUtil = new JdbcUtils();
        jdbcUtil.getConnection();
        String insertSql = "insert into mto_keyword (category,keyword,create_time) values (?,?,?)";
        String checkSql = "select count(1) count from mto_keyword where category = ? and keyword= ?";
        String updateSql = "update mto_keyword set create_time = ? where category = ? and keyword= ?";
        Date now = new Date();
        try {
            for (KeywordVo keyword:keywords){
                for (String s:keyword.getKeyword()){
                    List<Map<String, Object>> ret = jdbcUtil.findResult(checkSql,keyword.getCategory(),s);
                    if(Integer.parseInt(ret.get(0).get("count").toString())>0){
                        jdbcUtil.updateByPreparedStatement(updateSql,now,keyword.getCategory(),s);
                    }else{
                        jdbcUtil.updateByPreparedStatement(insertSql,keyword.getCategory(),s,now);
                    }
                }
            }
        } catch (SQLException e) {
            e.printStackTrace();
        } finally {
            jdbcUtil.releaseConn();
        }
    }

    public static void main(String [] args){
        try {
        MailUtils mailUtils = new MailUtils();
        // 设置环境信息
        Session session = mailUtils.getSession();

        // 创建邮件对象
        Message msg = new MimeMessage(session);
        msg.setSubject("爬虫通知");
        // 设置邮件内容
        msg.setText("这是一封由JavaMail发送的邮件！");
        // 设置发件人
        msg.setFrom(new InternetAddress("xmc0551@foxmail.com"));

        Transport transport = mailUtils.getTransport(session);

        // 发送邮件
        transport.sendMessage(msg, new Address[] {new InternetAddress("1170059620@qq.com")});
        // 关闭连接
        transport.close();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    //获取关键词目录
    public static List<CategoryVo> getCategory(){
        List<CategoryVo> cvs = new ArrayList<CategoryVo>();
        try {
            Document doc = Jsoup.connect("http://top.sogou.com").get();
            Elements as = doc.getElementsByClass("menu").get(0).getElementsByTag("a");
            for(int i=1;i<as.size();i++){
                CategoryVo cv = new CategoryVo();
                cv.setUrl("http://top.sogou.com/"+as.get(i).attr("href"));
                cv.setCategory(as.get(i).text());
                cvs.add(cv);
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
        return cvs;
    }

    //分目录获取搜狗关键词
    public static List<KeywordVo> getKeyword(){
        List<KeywordVo> keywords = new ArrayList<KeywordVo>();
        try {
            for(CategoryVo category:getCategory()){
                //System.out.println(category.getCategory());
                Document doc = Jsoup.connect(category.getUrl()).get();
                Elements lis = doc.getElementsByClass("pub-list").get(0).getElementsByTag("li");
                List<String> keyword = new ArrayList<String>();
                for(Element li:lis){
                    keyword.add(li.getElementsByClass("s2").get(0).getElementsByTag("a").get(0).text());
                }
                keywords.add(new KeywordVo(category.getCategory(),keyword));
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
        return keywords;
    }

    //去除重复
    public static List removeDuplicateWithOrder(List list) {
        Set set = new HashSet();
        List newList = new ArrayList();
        for (Iterator iter = list.iterator(); iter.hasNext();) {
            Object element = iter.next();
            if (set.add(element))
                newList.add(element);
        }
        list.clear();
        list.addAll(newList);
        return list;
    }

    //获取百度关键词目录
//    public static List<CategoryVo> getCategory(){
//        List<CategoryVo> cvs = new ArrayList<CategoryVo>();
//        try {
//            Document doc = Jsoup.connect("http://top.baidu.com/").get();
//            Elements hds = doc.getElementsByClass("hd");
//            for(Element hd : hds){
//                CategoryVo cv = new CategoryVo();
//                Element a=hd.getElementsByTag("a").get(0);
//                cv.setCategory(a.text());
//                cv.setUrl(a.attr("href").trim().replace("./","http://top.baidu.com/"));
//                cvs.add(cv);
//            }
//        } catch (IOException e) {
//            e.printStackTrace();
//        }
//        return cvs;
//    }

//    //分目录获取百度关键词
//    public static List<KeywordVo> getKeyword(){
//        List<KeywordVo> keywords = new ArrayList<KeywordVo>();
//        try {
//            for(CategoryVo category:getCategory()){
//                //System.out.println(category.getCategory());
//                Document doc = Jsoup.connect(category.getUrl()).get();
//                Elements boxConts = doc.getElementsByClass("box-cont");
//                List<String> keyword = new ArrayList<String>();
//                for(Element boxCont:boxConts){
//                    for(Element element:boxCont.getElementsByClass("list-title")){
//                        //System.out.println(element.text());
//                        keyword.add(element.attr("title"));
//                    }
//
//                    //System.out.println(boxCont.getElementsByClass("list-title")..text());
//                    //System.out.println(boxCont);
//                    //keyword.add(boxCont.getElementsByClass("list-title").attr("title"));
//                }
//                //System.out.println("------------");
//                keywords.add(new KeywordVo(category.getCategory(),removeDuplicateWithOrder(keyword)));
//            }
//        } catch (IOException e) {
//            e.printStackTrace();
//        }
//        return keywords;
//    }

}
